import java.io.*;
import java.util.*;

class Tokenizer implements Iterator<Token>
{
    public Tokenizer(String filename) throws FileNotFoundException
    {
		this.scanner = new Scanner(new File(filename));
        this.filename = filename;
    }
    
    public boolean hasNext()
    {
        return scanner.hasNext();
    }
    
    public Token next()
    {
        String word = scanner.next();
        Token token = new Token(filename, word, position);
        position++;
        return token;
    }
    
    public void remove()
    {
    }
    
    private Scanner scanner;
    private String filename;
    private int position = 0;
}
